# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


from pymongo import UpdateOne

from database import mongo_client
# useful for handling different item types with a single interface
from tutorial.items.fanqienovel_item import FanqienovelItem


class FanqienovelPipeline:
    cache: dict = {}

    def process_item(self, item: FanqienovelItem, spider):
        if not self.cache.get(item.book_name, []):
            self.cache[item.book_name] = []
        if len(self.cache[item.book_name]) < 1000:
            self.cache[item.book_name].append(
                UpdateOne(
                    {"chapter_index": item.chapter_index, "content": ""},
                    {"$set": item.model_dump(exclude={"book_name"})},
                    upsert=True
                )

            )
        else:
            self.insert_data(spider)

        return item

    def insert_data(self, spider):
        for book_name, items in self.cache.items():
            col = mongo_client.sync_client['ebooks'][book_name]
            col.create_index("chapter_index", unique=True)
            try:
                col.bulk_write(items)
            except Exception as e:
                spider.logger.error(f"插入数据失败:{str(e)}")
            else:
                spider.logger.info("插入了{}条数据".format(len(self.cache[book_name])))
            finally:
                self.cache[book_name] = []

    def open_spider(self, spider):
        """"""

    def close_spider(self, spider):
        if len(self.cache):
            self.insert_data(spider)
